{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5d244f26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.00342567,  0.01624046,  0.04334555, -0.0028494 ], dtype=float32)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('CartPole-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, done, _, info = self.env.step(action)\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            done = True\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b77c66d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAUPElEQVR4nO3db2xT570H8K/txIaQHIdAYy9LLLK7ajTjT7cAySm6WkVdsi6qxpoX28Tt0l5EVeag0lRoi9RS2m1KxXTVrRsNbzbgDaVKJVo1on+yQIMqDIGwXIUAue0VLBnFdim1nYTGTnx+9wU3Z3UJNCbBT4y/H+lInOd5bP/OSfzl+Dw5xxYRERARKWBVXQARZS8GEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKaMsgHbu3IlFixZhzpw5qKqqQldXl6pSiEgRJQH0+uuvo7GxEc8//zxOnTqF5cuXo6amBqFQSEU5RKSIRcXFqFVVVVi5ciX+/Oc/AwAMw0BZWRk2b96MX//61+kuh4gUyUn3C8bjcXR3d6Opqclss1qt8Hq98Pv9kz4mFoshFouZ64Zh4MqVK1iwYAEsFsttr5mIUiMiGBoaQklJCazWG3/QSnsAXb58GYlEAi6XK6nd5XLh3Llzkz6mubkZL7zwQjrKI6IZNDg4iNLS0hv2pz2AbkVTUxMaGxvN9UgkAo/Hg8HBQWiaprAyIppMNBpFWVkZCgoKbjou7QG0cOFC2Gw2BIPBpPZgMAi32z3pYxwOBxwOx3XtmqYxgIhmsa87RZL2WTC73Y7Kykp0dHSYbYZhoKOjA7qup7scIlJIyUewxsZG1NfXY8WKFVi1ahX+8Ic/YGRkBI8//riKcohIESUB9NOf/hSffvoptm3bhkAggHvvvRfvvvvudSemiejOpuTvgKYrGo3C6XQiEonwHBDRLDTV9yivBSMiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyKQfQkSNH8PDDD6OkpAQWiwVvvvlmUr+IYNu2bfjGN76BuXPnwuv14qOPPkoac+XKFaxfvx6apqGwsBAbNmzA8PDwtDaEiDJPygE0MjKC5cuXY+fOnZP279ixA6+88gp27dqF48ePY968eaipqcHo6Kg5Zv369ejr60N7ezva2tpw5MgRPPHEE7e+FUSUmWQaAMiBAwfMdcMwxO12y+9//3uzLRwOi8PhkNdee01ERM6cOSMA5MSJE+aYd955RywWi1y8eHFKrxuJRASARCKR6ZRPRLfJVN+jM3oO6Pz58wgEAvB6vWab0+lEVVUV/H4/AMDv96OwsBArVqwwx3i9XlitVhw/fnzS543FYohGo0kLEWW+GQ2gQCAAAHC5XEntLpfL7AsEAiguLk7qz8nJQVFRkTnmq5qbm+F0Os2lrKxsJssmIkUyYhasqakJkUjEXAYHB1WXREQzYEYDyO12AwCCwWBSezAYNPvcbjdCoVBS//j4OK5cuWKO+SqHwwFN05IWIsp8MxpA5eXlcLvd6OjoMNui0SiOHz8OXdcBALquIxwOo7u72xxz6NAhGIaBqqqqmSyHiGa5nFQfMDw8jI8//thcP3/+PHp6elBUVASPx4MtW7bgt7/9Le6++26Ul5fjueeeQ0lJCdatWwcAuOeee/DDH/4QGzduxK5duzA2NoaGhgb87Gc/Q0lJyYxtGBFlgFSn1w4fPiwArlvq6+tF5NpU/HPPPScul0scDoc88MAD0t/fn/Qcn332mfz85z+X/Px80TRNHn/8cRkaGprxKT4iUmOq71GLiIjC/Lsl0WgUTqcTkUiE54OIZqGpvkczYhaMiO5MDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhImZS/lodopomRQHjgNIyxUbNtnutbmKPdpbAqSgcGEClnjMcxcPR1xIcum23l9z/GAMoC/AhGyhmJccBIqC6DFGAAkXKSGIeIoboMUoABRMoZBgMoWzGASLlE7CqM8bF/NVissOY61BVEacMAIuXGvogmzYBZc+xwFCxUWBGlCwOIZh2LxQKLjRO02SClAGpubsbKlStRUFCA4uJirFu3Dv39/UljRkdH4fP5sGDBAuTn56Ourg7BYDBpzMDAAGpra5GXl4fi4mJs3boV4+Pj098aujNYLLDaclVXQWmQUgB1dnbC5/Ph2LFjaG9vx9jYGNauXYuRkRFzzNNPP423334bra2t6OzsxCeffIJHHnnE7E8kEqitrUU8HsfRo0exd+9e7NmzB9u2bZu5raKMZrFYGUDZQqYhFAoJAOns7BQRkXA4LLm5udLa2mqOOXv2rAAQv98vIiIHDx4Uq9UqgUDAHNPS0iKapkksFpvS60YiEQEgkUhkOuXTLPH5hf+Wrl0bzeXU3kaJX42qLoumYarv0WmdA4pEIgCAoqIiAEB3dzfGxsbg9XrNMYsXL4bH44Hf7wcA+P1+LF26FC6XyxxTU1ODaDSKvr6+SV8nFoshGo0mLXTnMBJjXz+I7ki3HECGYWDLli1YvXo1lixZAgAIBAKw2+0oLCxMGutyuRAIBMwxXw6fif6Jvsk0NzfD6XSaS1lZ2a2WTbOQMR5XXQIpcssB5PP5cPr0aezfv38m65lUU1MTIpGIuQwODt7216T0YQBlr1ua62xoaEBbWxuOHDmC0tJSs93tdiMejyMcDicdBQWDQbjdbnNMV1dX0vNNzJJNjPkqh8MBh4N/mHanYgBlr5SOgEQEDQ0NOHDgAA4dOoTy8vKk/srKSuTm5qKjo8Ns6+/vx8DAAHRdBwDouo7e3l6EQiFzTHt7OzRNQ0VFxXS2hTLUaDj5o7d93nzOgmWJlI6AfD4f9u3bh7feegsFBQXmORun04m5c+fC6XRiw4YNaGxsRFFRETRNw+bNm6HrOqqrqwEAa9euRUVFBR599FHs2LEDgUAAzz77LHw+H49ystT46EjSes7cAlisNkXVUDqlFEAtLS0AgPvvvz+pfffu3XjssccAAC+//DKsVivq6uoQi8VQU1ODV1991Rxrs9nQ1taGTZs2Qdd1zJs3D/X19XjxxRentyV0x7DacgGLRXUZlAYWERHVRaQqGo3C6XQiEolA0zTV5dA0ffz+Lnx+/pS5XvRvK/GtNf/Jo6AMNtX3KK8Fo1nHYssFLPzVzAb8KZNSInLdvYCsNh75ZAsGEKklBiRx/YXIFp4DygoMIFJKxLh2T2jKSgwgUkoMA2IwgLIVA4jUEuP6v4TmCeiswZ80KZWIjyL2pe8DA4C8om8qqobSjQFESokYECN5Fsxmn6uoGko3BhDNMhZYc+yqi6A0YQDRrMMAyh4MIJp1GEDZgwFESomRAPClyxEtgMXKX8tswZ80KWUkxpLyh7ILA4iUuvaVzEygbMUAIqUkEUcG3hGGZggDiJQaHx0BvnQ1vMWac+12HJQVGECkVCz66f+fiL4md24BcufyJnPZggFEs4rFauOdELMIA4hmFQZQdmEA0azCAMouDCBSRkQgX5mCt1isDKAswgAipa67HavFwtuxZhEGECnFr2XObgwgUooBlN1S+mZUolRdvXoV8fjkISMiGL06nNSWSCQQDodveB4oPz8fOTn8tb1T8CdJt9X27duxb9++SftsVgu2/0c1vuuZb7Yd/rAL//XMchiTXJ1htVrx1ltv4Xvf+97tKpfSjAFEt1U4HMbFixcn7cuxWWGx2HF25D4MJeaj1NGPwWAf/vnPi5Nenmq1Wm94NEWZKaVzQC0tLVi2bBk0TYOmadB1He+8847ZPzo6Cp/PhwULFiA/Px91dXUIBoNJzzEwMIDa2lrk5eWhuLgYW7duxfg4v5YlG4lYcGZkNf4x+l1cGfsm+ob/HQMjJbw2PoukFEClpaV46aWX0N3djZMnT2LNmjX48Y9/jL6+PgDA008/jbfffhutra3o7OzEJ598gkceecR8fCKRQG1tLeLxOI4ePYq9e/diz5492LZt28xuFWUEgQXDifkArk27J5CLz7/IU1sUpVVKH8EefvjhpPXf/e53aGlpwbFjx1BaWoq//OUv2LdvH9asWQMA2L17N+655x4cO3YM1dXVeP/993HmzBn87W9/g8vlwr333ovf/OY3+NWvfoXt27fDbuetOLOJxWLgm46PICiHARvm2SKYh3+oLovS6JbPASUSCbS2tmJkZAS6rqO7uxtjY2Pwer3mmMWLF8Pj8cDv96O6uhp+vx9Lly6Fy+Uyx9TU1GDTpk3o6+tL+eTiuXPnkJ+ff6ubQGnw+eef37DPMAwc87+GhOMEriacuMs+iP/53zM3fb4LFy6goKBgpsukGTY8PPz1g3ALAdTb2wtd1zE6Oor8/HwcOHAAFRUV6Onpgd1uR2FhYdJ4l8uFQCAAAAgEAknhM9E/0XcjsVgMsVjMXI9GowCASCTC80ez3M1OGosAbUfPATg35ecbGhpCOByefmF0W42MjExpXMoB9J3vfAc9PT2IRCJ44403UF9fj87OzpQLTEVzczNeeOGF69qrqqqgabx3zGz21f9wpmvp0qWoqqqa0eekmTdxkPB1Uv5LaLvdjm9/+9uorKxEc3Mzli9fjj/+8Y9wu92Ix+PX/e8UDAbhdrsBAG63+7pZsYn1iTGTaWpqQiQSMZfBwcFUyyaiWWjal2IYhoFYLIbKykrk5uaio6PD7Ovv78fAwAB0XQcA6LqO3t5ehEIhc0x7ezs0TUNFRcUNX8PhcJhT/xMLEWW+lD6CNTU14aGHHoLH48HQ0BD27duHDz74AO+99x6cTic2bNiAxsZGFBUVQdM0bN68Gbquo7q6GgCwdu1aVFRU4NFHH8WOHTsQCATw7LPPwufzweFw3JYNJKLZK6UACoVC+MUvfoFLly7B6XRi2bJleO+99/Dggw8CAF5++WVYrVbU1dUhFouhpqYGr776qvl4m82GtrY2bNq0CbquY968eaivr8eLL744s1tFs8acOXNm7IjVZrPxOrA7jEUy8DtRotEonE4nIpEIP47NcleuXJnyjMhUFBcX82g5A0z1Pcr/Tui2KioqQlFRkeoyaJbi/YCISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMjmqC7gVIgIAiEajiishoslMvDcn3qs3kpEB9NlnnwEAysrKFFdCRDczNDQEp9N5w/6MDKCioiIAwMDAwE03jpJFo1GUlZVhcHAQmqapLicjcJ/dGhHB0NAQSkpKbjouIwPIar126srpdPKX4hZomsb9liLus9RN5eCAJ6GJSBkGEBEpk5EB5HA48Pzzz8PhcKguJaNwv6WO++z2ssjXzZMREd0mGXkERER3BgYQESnDACIiZRhARKRMRgbQzp07sWjRIsyZMwdVVVXo6upSXZIyzc3NWLlyJQoKClBcXIx169ahv78/aczo6Ch8Ph8WLFiA/Px81NXVIRgMJo0ZGBhAbW0t8vLyUFxcjK1bt2J8fDydm6LMSy+9BIvFgi1btpht3GdpIhlm//79Yrfb5a9//av09fXJxo0bpbCwUILBoOrSlKipqZHdu3fL6dOnpaenR370ox+Jx+OR4eFhc8yTTz4pZWVl0tHRISdPnpTq6mq57777zP7x8XFZsmSJeL1e+fvf/y4HDx6UhQsXSlNTk4pNSquuri5ZtGiRLFu2TJ566imznfssPTIugFatWiU+n89cTyQSUlJSIs3NzQqrmj1CoZAAkM7OThERCYfDkpubK62treaYs2fPCgDx+/0iInLw4EGxWq0SCATMMS0tLaJpmsRisfRuQBoNDQ3J3XffLe3t7fKDH/zADCDus/TJqI9g8Xgc3d3d8Hq9ZpvVaoXX64Xf71dY2ewRiUQA/OuC3e7uboyNjSXts8WLF8Pj8Zj7zO/3Y+nSpXC5XOaYmpoaRKNR9PX1pbH69PL5fKitrU3aNwD3WTpl1MWoly9fRiKRSPqhA4DL5cK5c+cUVTV7GIaBLVu2YPXq1ViyZAkAIBAIwG63o7CwMGmsy+VCIBAwx0y2Tyf67kT79+/HqVOncOLEiev6uM/SJ6MCiG7O5/Ph9OnT+PDDD1WXMqsNDg7iqaeeQnt7O+bMmaO6nKyWUR/BFi5cCJvNdt1sRDAYhNvtVlTV7NDQ0IC2tjYcPnwYpaWlZrvb7UY8Hkc4HE4a/+V95na7J92nE313mu7uboRCIXz/+99HTk4OcnJy0NnZiVdeeQU5OTlwuVzcZ2mSUQFkt9tRWVmJjo4Os80wDHR0dEDXdYWVqSMiaGhowIEDB3Do0CGUl5cn9VdWViI3Nzdpn/X392NgYMDcZ7quo7e3F6FQyBzT3t4OTdNQUVGRng1JowceeAC9vb3o6ekxlxUrVmD9+vXmv7nP0kT1WfBU7d+/XxwOh+zZs0fOnDkjTzzxhBQWFibNRmSTTZs2idPplA8++EAuXbpkLlevXjXHPPnkk+LxeOTQoUNy8uRJ0XVddF03+yemlNeuXSs9PT3y7rvvyl133ZVVU8pfngUT4T5Ll4wLIBGRP/3pT+LxeMRut8uqVavk2LFjqktSBsCky+7du80xX3zxhfzyl7+U+fPnS15envzkJz+RS5cuJT3PhQsX5KGHHpK5c+fKwoUL5ZlnnpGxsbE0b406Xw0g7rP04O04iEiZjDoHRER3FgYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkzP8Bzey7JgcY5qAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.figure(figsize=(3, 3))\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f489de60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env.observation_space= Box([-4.8000002e+00 -3.4028235e+38 -4.1887903e-01 -3.4028235e+38], [4.8000002e+00 3.4028235e+38 4.1887903e-01 3.4028235e+38], (4,), float32)\n",
      "env.action_space= Discrete(2)\n",
      "state= [ 0.00609916 -0.02172713  0.0349807   0.03277298]\n",
      "action= 0\n",
      "next_state= [ 0.00566462 -0.2173328   0.03563616  0.33628422]\n",
      "reward= 1.0\n",
      "done= False\n",
      "info= {}\n"
     ]
    }
   ],
   "source": [
    "#认识游戏环境\n",
    "def test_env():\n",
    "    print('env.observation_space=', env.observation_space)\n",
    "    print('env.action_space=', env.action_space)\n",
    "\n",
    "    state = env.reset()\n",
    "    action = env.action_space.sample()\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "\n",
    "    print('state=', state)\n",
    "    print('action=', action)\n",
    "    print('next_state=', next_state)\n",
    "    print('reward=', reward)\n",
    "    print('done=', done)\n",
    "    print('info=', info)\n",
    "\n",
    "\n",
    "test_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6d5c572d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.a2c.a2c.A2C at 0x7f3a7a3be760>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3 import A2C\n",
    "from stable_baselines3.common.env_util import make_vec_env\n",
    "\n",
    "#初始化模型\n",
    "model = A2C(\n",
    "    policy='MlpPolicy',\n",
    "    env=make_vec_env(MyWrapper, n_envs=8),  #使用N个环境同时训练\n",
    "    learning_rate=1e-3,\n",
    "    n_steps=5,  #运行N步后执行更新,batch_size=n_steps*环境数量\n",
    "    gamma=0.9,\n",
    "    verbose=0)\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22afd2bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/stable_baselines3/common/evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(19.65, 6.4519376934375305)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "75e0b593",
   "metadata": {},
   "outputs": [],
   "source": [
    "# #训练\n",
    "# model.learn(200_0000, progress_bar=True)\n",
    "\n",
    "# #保存模型\n",
    "# model.save('save/2.A2C.CartPole')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cdaaf12a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(199.4, 1.5620499351813306)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#加载模型\n",
    "model = A2C.load('save/2.A2C.CartPole')\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "414e1478",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAATuklEQVR4nO3db2xT97kH8K+dP4aQHIckjd3cxBfuVpWm/NkWIJzxYtNwybpoWtZI6ybUZRWiKnNQaSakRmpBoEqp6Iu2bDS8mAa9Lzp6U4lOzU1bRYYGVbikhEYKKaSbLruJANsFmmMnJbYTP/dFb85qCDQmwT/b+X6kg+rze2w/58D59vh3cmKLiAiIiBSwqm6AiBYuBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESmjLIAOHjyIZcuWYdGiRaitrUVvb6+qVohIESUB9NZbb6GlpQV79uzB2bNnsWbNGtTV1SEYDKpoh4gUsai4GbW2thbr1q3Dn/70JwBAPB5HVVUVduzYgeeeey7V7RCRIrmpfsNoNIq+vj60traa66xWK9xuN3w+34zPiUQiiEQi5uN4PI7r16+jtLQUFovlnvdMRMkREYTDYVRUVMBqvf0HrZQH0NWrVzE1NQWHw5Gw3uFw4MKFCzM+p62tDXv37k1Fe0Q0j0ZGRlBZWXnb8ZQH0N1obW1FS0uL+dgwDLhcLoyMjEDTNIWdEdFMQqEQqqqqUFRUdMe6lAdQWVkZcnJyEAgEEtYHAgE4nc4Zn2Oz2WCz2W5Zr2kaA4gojX3bFEnKr4Ll5+ejpqYGXq/XXBePx+H1eqHreqrbISKFlHwEa2lpQVNTE9auXYv169fj1Vdfxfj4OJ588kkV7RCRIkoC6PHHH8cXX3yB3bt3w+/343vf+x7ef//9WyamiSi7Kfk5oLkKhUKw2+0wDINzQERpaLbHKO8FIyJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTJJB9DJkyfx85//HBUVFbBYLHjnnXcSxkUEu3fvxv3334/FixfD7Xbj73//e0LN9evXsWXLFmiahuLiYmzduhVjY2Nz2hAiyjxJB9D4+DjWrFmDgwcPzji+f/9+HDhwAIcOHcLp06exZMkS1NXVYWJiwqzZsmULBgcH0d3djc7OTpw8eRJPPfXU3W8FEWUmmQMAcuzYMfNxPB4Xp9MpL7/8srludHRUbDab/PWvfxURkc8++0wAyCeffGLWvPfee2KxWOTSpUuzel/DMASAGIYxl/aJ6B6Z7TE6r3NAFy9ehN/vh9vtNtfZ7XbU1tbC5/MBAHw+H4qLi7F27Vqzxu12w2q14vTp0zO+biQSQSgUSliIKPPNawD5/X4AgMPhSFjvcDjMMb/fj/Ly8oTx3NxclJSUmDU3a2trg91uN5eqqqr5bJuIFMmIq2Ctra0wDMNcRkZGVLdERPNgXgPI6XQCAAKBQML6QCBgjjmdTgSDwYTxyclJXL9+3ay5mc1mg6ZpCQsRZb55DaDly5fD6XTC6/Wa60KhEE6fPg1d1wEAuq5jdHQUfX19Zs3x48cRj8dRW1s7n+0QUZrLTfYJY2Nj+Mc//mE+vnjxIvr7+1FSUgKXy4WdO3fixRdfxAMPPIDly5fjhRdeQEVFBRoaGgAADz30EH76059i27ZtOHToEGKxGJqbm/HrX/8aFRUV87ZhRJQBkr28duLECQFwy9LU1CQiX1+Kf+GFF8ThcIjNZpNNmzbJ0NBQwmtcu3ZNfvOb30hhYaFomiZPPvmkhMPheb/ER0RqzPYYtYiIKMy/uxIKhWC322EYBueDiNLQbI/RjLgKRkTZiQFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKZP01/LQvXfj+mV8de1f3/6au6gQWmU1LBaLwq6I5h8DKA19+b/9uNT7jvl4yX3LoP3bQwADiLIMP4JlAPnGn0TZhAGUCUSQgV/fRvStGEAZgeFD2YkBlDEYQpR9GECZQIT5Q1mJAZQBOAlN2SqpAGpra8O6detQVFSE8vJyNDQ0YGhoKKFmYmICHo8HpaWlKCwsRGNjIwKBQELN8PAw6uvrUVBQgPLycuzatQuTk5Nz35psxUloylJJBVBPTw88Hg8+/vhjdHd3IxaLYfPmzRgfHzdrnn32Wbz77rvo6OhAT08PLl++jMcee8wcn5qaQn19PaLRKE6dOoU33ngDR44cwe7du+dvq7IOw4eylMxBMBgUANLT0yMiIqOjo5KXlycdHR1mzfnz5wWA+Hw+ERHp6uoSq9Uqfr/frGlvbxdN0yQSiczqfQ3DEABiGMZc2k9bl87+t/Qe2mYuA/+1RyajN1S3RTRrsz1G5zQHZBgGAKCkpAQA0NfXh1gsBrfbbdasWLECLpcLPp8PAODz+bBq1So4HA6zpq6uDqFQCIODgzO+TyQSQSgUSlgWEhHwJIiy0l0HUDwex86dO7Fx40asXLkSAOD3+5Gfn4/i4uKEWofDAb/fb9Z8M3ymx6fHZtLW1ga73W4uVVVVd9t2hhIIE4iy0F0HkMfjwblz53D06NH57GdGra2tMAzDXEZGRr79SdlE8P+nQUTZ5a5uRm1ubkZnZydOnjyJyspKc73T6UQ0GsXo6GjCWVAgEIDT6TRrent7E15v+irZdM3NbDYbbDbb3bSaJRg+lJ2SOgMSETQ3N+PYsWM4fvw4li9fnjBeU1ODvLw8eL1ec93Q0BCGh4eh6zoAQNd1DAwMIBgMmjXd3d3QNA3V1dVz2ZbsxUkgylJJnQF5PB68+eab+Nvf/oaioiJzzsZut2Px4sWw2+3YunUrWlpaUFJSAk3TsGPHDui6jg0bNgAANm/ejOrqajzxxBPYv38//H4/nn/+eXg8ngV+lnN7Yv5BlF2SCqD29nYAwI9//OOE9YcPH8bvfvc7AMArr7wCq9WKxsZGRCIR1NXV4fXXXzdrc3Jy0NnZie3bt0PXdSxZsgRNTU3Yt2/f3LYkq3ESmrKTRSTzZjdDoRDsdjsMw4CmaarbmXeXP+1K+IVkNu0+PNTwHPIWF6lriigJsz1GeS8YESnDAMoEnISmLMUAygCchKZsxQDKBMJJaMpODKCMIPxJaMpKDKA0ZLEk/rWIMIAoOzGA0tAiuwMWa475ePJGGLGJsMKOiO4NBlAaslgTfz5UJM4zIMpKDKB0xG9ApQWCAZSGvv4OeIYQZT8GUDriGRAtEAygtMQAooWBAZSGLDwDogWCAZSOLBaeBNGCwABKQzwDooWCAZSWGEC0MDCA0hEvw9MCwQBKQzffC0aUrfgvPS3x7IcWBgZQOuIkNC0QDKA0xKtgtFAwgNIRA4gWCAZQGrLAwlkgWhAYQOnIYuFZEC0ISX0zKs0PEUE4HEY8Hp9xPDI2hpu/LzIcDiOWNzpjvdVqRVFREeeOKOMwgBSIxWJoaGjA559/PuN4ZdkSvLx1I2x5X//1xOOT+NXjv8L/XAnNWH///ffD6/Vm5bfEUnZjACkgIggGg7h06dKM47mTdoQml+LSxHrEkYN/t53FF8EvcOnS1RnrLRbLbc+miNJZUnNA7e3tWL16NTRNg6Zp0HUd7733njk+MTEBj8eD0tJSFBYWorGxEYFAIOE1hoeHUV9fj4KCApSXl2PXrl2YnJycn63JEhNTi9Af2gR/9DsIRpehP+zGV1P8XnjKPkkFUGVlJV566SX09fXhzJkz+MlPfoJf/OIXGBwcBAA8++yzePfdd9HR0YGenh5cvnwZjz32mPn8qakp1NfXIxqN4tSpU3jjjTdw5MgR7N69e363KsPF4rm4ES80H0fiSxATm8KOiO4RmaOlS5fKn//8ZxkdHZW8vDzp6Ogwx86fPy8AxOfziYhIV1eXWK1W8fv9Zk17e7tomiaRSGTW72kYhgAQwzDm2r4SExMT8vDDD09/4fsti7N0qbz24n/KvhdPy94XT8uBtrflO5UVt62vrKyUL7/8UvVmEZlme4ze9RzQ1NQUOjo6MD4+Dl3X0dfXh1gsBrfbbdasWLECLpcLPp8PGzZsgM/nw6pVq+BwOMyauro6bN++HYODg/j+97+fVA8XLlxAYWHhtxemmVgshkgkctvxsa/GcerEq7g29R+ISw7K8y/iy9CXd3y9oaEhFBXxYxqlh7GxsVnVJR1AAwMD0HUdExMTKCwsxLFjx1BdXY3+/n7k5+ejuLg4od7hcMDv9wMA/H5/QvhMj0+P3U4kEkk4YEOhr68GGYaRkfNHsVgMU1NTtx0fuxHFW96zAM7O6vXi8ThCodAdX5MolcbHx2dVl3QAPfjgg+jv74dhGHj77bfR1NSEnp6epBtMRltbG/bu3XvL+tra2oy89ByJRFBQUDBvr2ez2bBu3bpbwp9IlemThG+T9E9C5+fn47vf/S5qamrQ1taGNWvW4LXXXoPT6UQ0GsXo6GhCfSAQgNPpBAA4nc5bropNP56umUlraysMwzCXkZGRZNsmojQ051sx4vE4IpEIampqkJeXB6/Xa44NDQ1heHgYuq4DAHRdx8DAAILBoFnT3d0NTdNQXV192/ew2Wzmpf/phYgyX1IfwVpbW/Hoo4/C5XIhHA7jzTffxIcffogPPvgAdrsdW7duRUtLC0pKSqBpGnbs2AFd17FhwwYAwObNm1FdXY0nnngC+/fvh9/vx/PPPw+PxwObjZeZiRaapAIoGAzit7/9La5cuQK73Y7Vq1fjgw8+wCOPPAIAeOWVV2C1WtHY2IhIJIK6ujq8/vrr5vNzcnLQ2dmJ7du3Q9d1LFmyBE1NTdi3b9/8blUGKCwsnLczOd4HRpnKInLTXY8ZIBQKwW63wzCMjPw4JiIIBAKIxWLz8no5OTlwOp2wWvnLDSg9zPYY5b1gClgsljtOuhMtFPxfJhEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlImV3UDd0NEAAChUEhxJ0Q0k+ljc/pYvZ2MDKBr164BAKqqqhR3QkR3Eg6HYbfbbzuekQFUUlICABgeHr7jxlGiUCiEqqoqjIyMQNM01e1kBO6zuyMiCIfDqKiouGNdRgaQ1fr11JXdbuc/irugaRr3W5K4z5I3m5MDTkITkTIMICJSJiMDyGazYc+ePbDZbKpbySjcb8njPru3LPJt18mIiO6RjDwDIqLswAAiImUYQESkDAOIiJTJyAA6ePAgli1bhkWLFqG2tha9vb2qW1Kmra0N69atQ1FREcrLy9HQ0IChoaGEmomJCXg8HpSWlqKwsBCNjY0IBAIJNcPDw6ivr0dBQQHKy8uxa9cuTE5OpnJTlHnppZdgsViwc+dOcx33WYpIhjl69Kjk5+fLX/7yFxkcHJRt27ZJcXGxBAIB1a0pUVdXJ4cPH5Zz585Jf3+//OxnPxOXyyVjY2NmzdNPPy1VVVXi9XrlzJkzsmHDBvnhD39ojk9OTsrKlSvF7XbLp59+Kl1dXVJWViatra0qNimlent7ZdmyZbJ69Wp55plnzPXcZ6mRcQG0fv168Xg85uOpqSmpqKiQtrY2hV2lj2AwKACkp6dHRERGR0clLy9POjo6zJrz588LAPH5fCIi0tXVJVarVfx+v1nT3t4umqZJJBJJ7QakUDgclgceeEC6u7vlRz/6kRlA3Gepk1EfwaLRKPr6+uB2u811VqsVbrcbPp9PYWfpwzAMAP+6Ybevrw+xWCxhn61YsQIul8vcZz6fD6tWrYLD4TBr6urqEAqFMDg4mMLuU8vj8aC+vj5h3wDcZ6mUUTejXr16FVNTUwl/6QDgcDhw4cIFRV2lj3g8jp07d2Ljxo1YuXIlAMDv9yM/Px/FxcUJtQ6HA36/36yZaZ9Oj2Wjo0eP4uzZs/jkk09uGeM+S52MCiC6M4/Hg3PnzuGjjz5S3UpaGxkZwTPPPIPu7m4sWrRIdTsLWkZ9BCsrK0NOTs4tVyMCgQCcTqeirtJDc3MzOjs7ceLECVRWVprrnU4notEoRkdHE+q/uc+cTueM+3R6LNv09fUhGAziBz/4AXJzc5Gbm4uenh4cOHAAubm5cDgc3GcpklEBlJ+fj5qaGni9XnNdPB6H1+uFrusKO1NHRNDc3Ixjx47h+PHjWL58ecJ4TU0N8vLyEvbZ0NAQhoeHzX2m6zoGBgYQDAbNmu7ubmiahurq6tRsSApt2rQJAwMD6O/vN5e1a9diy5Yt5n9zn6WI6lnwZB09elRsNpscOXJEPvvsM3nqqaekuLg44WrEQrJ9+3ax2+3y4YcfypUrV8zlq6++Mmuefvppcblccvz4cTlz5ozoui66rpvj05eUN2/eLP39/fL+++/Lfffdt6AuKX/zKpgI91mqZFwAiYj88Y9/FJfLJfn5+bJ+/Xr5+OOPVbekDIAZl8OHD5s1N27ckN///veydOlSKSgokF/+8pdy5cqVhNf55z//KY8++qgsXrxYysrK5A9/+IPEYrEUb406NwcQ91lq8NdxEJEyGTUHRETZhQFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEp83/5HJzSiCk/zAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200.0 200 [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0]\n"
     ]
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "def test():\n",
    "    state = env.reset()\n",
    "    reward_sum = []\n",
    "    over = False\n",
    "    while not over:\n",
    "        action, _ = model.predict(state)\n",
    "        action = action.item()\n",
    "        state, reward, over, _ = env.step(action)\n",
    "        reward_sum.append(reward)\n",
    "\n",
    "        if len(reward_sum) % 5 == 0:\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    print(sum(reward_sum), len(reward_sum), reward_sum)\n",
    "\n",
    "\n",
    "test()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
